########################################################################
from logutilitie import logutilitie
import re
import urllib.request
import configmap
import os
import time
from publishutilitie import publishutilitie
class treasure:
    """"""
    decode = ''
    log = None
    readUrlList = []
    downInfoHtmlList = []
    downCurrentUrlList= []
    downPictureHtmlList = []
    scriptTreeRule = """<SCRIPT(?:[^<]+|<(?!/SCRIPT>))*</SCRIPT>"""
    pictreeListRule = "pictree.=.\[(.*?)\]"
    hostUrlRule = "^(.*)(?<=/)"
    imgSrcForIdRuel = """<img.*?\sid=['"]disp['"].*?\ssrc=['"](.*?)['"].*?>"""
    rule4IndexRule = """<a[^>]*href=["|'](?P<id_1>\d*)/(?P<id_2>\d*).html["|'].*?>(.*?)</a>"""
    downLogFile = __name__+'down.log'
    downAsIndexLogFile = 'as_index.log'
    downAsRule4LogFile = 'as_rule_4.log'
    splitLogFileDir = 'split'
    splitLogFileToName = 'split_url_%d.log'
    splitLogFIleForNumberRule = 'split_url_(.*\d).log'
    picHtmlLogFileDir = 'pic_html'
    picLogFileDir = 'pic'
    stepNum = 0
    #----------------------------------------------------------------------
    def __init__(self,decode,logInputDirName,clearDir = False):
        """Constructor"""
        self.decode = decode
        self.log = logutilitie(logInputDirName,'',clearDir)
        self.downCurrentUrlList = self.log.readLogToList('current_url_list.log')
        #self.readUrlList = self.log.readLogToList(self.downLogFile)
    #----------------------------------------------------------------------
    def run(self,startLogNumber = 0):
        """"""
        #self.getTreasureInfos()
        #self.splitFileForFileName(self.downAsIndexLogFile)
        #self.getTreasureUrlsSplitFile()
        self.getPicUrl(startLogNumber)
    
    #----------------------------------------------------------------------
    def getTreasureInfos(self):
        """"""
        self.ownInfoHtmlList = self.log.readLogToList(self.downAsRule4LogFile)
        for url in self.ownInfoHtmlList :
            m = re.match(self.hostUrlRule,url)
            currentUrl = m.group(0)
            urlLog = re.match('(.*?)\\n',urlInfo)
            if(len(urlLog.group(0)) != 0) :
                print(url)
                content = self.getHtmlFromUrl(url)
                if(len(content) > 0) : 
                    self.downCurrentUrlList = self.downCurrentUrlList + [url]
                    self.log.wirteLog('current_url_list.log')
                    self.stepNum = len(self.downCurrentUrlList)
                    logFileName = os.path.join(self.splitLogFileDir,self.splitLogFileToName % self.stepNum)
                    rule4IndexUrlList = re.findall(self.rule4IndexRule,content)
                    for id1,id2,name in rule4IndexUrlList:
                        self.log.wirteLog(logFileName,"%s%s/%s.html%s%s" % tuple([currentUrl,id1,id2,configmap.splitLogString,name]))
            else:
                print('end url:',url)
            #TreasureInfoList = self.getTreasureInfo(content)
    #----------------------------------------------------------------------
    def splitFileForFileName(self,fileName):
        print("start split file",self.downAsIndexLogFile)
        asRule4List = self.log.readLogToList(self.downAsRule4LogFile)
        logList = self.log.readLogToList(self.downAsIndexLogFile)
        print('log list number :',len(logList))
        logNum = 1
        for log in logList :
            if(logNum % 10000 == 0):
                print(logNum)
                logNum = 1
            urlInfo = log.split(configmap.splitLogString)
            r1 = '\d*/\d*\.html'
            r2 = 'index.html\n'
            asRule4Url = re.sub(r1,r2,urlInfo[0])
            stepNum = asRule4List.index(asRule4Url)
            logName = self.splitLogFileToName % stepNum
            fileName = "%s/%s" % tuple([self.splitLogFileDir,logName])
            self.log.wirteLog(fileName,urlInfo[0])
            logNum += 1
    #----------------------------------------------------------------------
    
    #----------------------------------------------------------------------
    def getTreasureInfo(self,content,url):
        """"""
        
    #----------------------------------------------------------------------
    def getTreasureUrlsSplitFile(self):
        """"""
        picHtmlDir = os.path.join(self.log.LogFilePath,self.picHtmlLogFileDir)
        for parentDir,dirNames,picHtmlFileNames in os.walk(picHtmlDir):
            print('read pic html dir : ',picHtmlDir )
        splitDir = os.path.join(self.log.LogFilePath,self.splitLogFileDir)
        picHtmlFileNames = []
        for parentDir,dirNames,fileNames in os.walk(splitDir):
            print('read split dir : ',splitDir )
            for fileName in fileNames :
                if(fileName in picHtmlFileNames):
                    print("bak end :",fileName)
                    continue
                else:
                    print("create :",fileName,publishutilitie.getLocationTime())
                asIndexUrlLog = self.log.readLogToList("%s/%s" % tuple([self.splitLogFileDir,fileName]))
                PicUrlLogFile = ("%s/%s" % tuple([self.picHtmlLogFileDir,fileName]))
                for urlInfo in asIndexUrlLog :
                    url = urlInfo.split(configmap.splitLogString)
                    if(url[0] != '\n'):
                        picUrl = self.getTreasureUrls(url[0])
                        self.log.wirteLog(PicUrlLogFile,picUrl)
        
        
    #----------------------------------------------------------------------
    def getTreasureUrls(self,downUrl):
        """"""
        content = self.getHtmlFromUrl(downUrl)
        m = re.match(self.hostUrlRule,downUrl)
        downRelativeUrl = m.group(0)
        picTreeStrList = re.findall(self.pictreeListRule,content)
        picList = []
        returnUrl = []
        for picTreeStr  in picTreeStrList :
            self.log.wirteLog('picTreeStr.log',picTreeStr)
            picTreeList = re.sub("'","",picTreeStr)
            picList = picTreeList.split(",")
            for url in picList:
                returnUrl.append('%s%s' % tuple([downRelativeUrl,url]))
        return returnUrl
    
    def getPicUrl(self,startLogNumber=0):
        if(type(startLogNumber) != int):
            startLogNumber = 0
        maxLogNumber = startLogNumber + 500
        picDir = os.path.join(self.log.LogFilePath,self.picLogFileDir)
        picFileNames = []
        for parentDir,dirNames,picFileNames in os.walk(picDir):
            print('start read pic dir : picFileNames | startLogNumber',startLogNumber )
        picHtmlDir = os.path.join(self.log.LogFilePath,self.picHtmlLogFileDir)
        for parentDir,dirNames,picHtmlFileNames in os.walk(picHtmlDir):
            print('getPicUrl read dir : ', picHtmlDir )
            picHtmlFileNames.reverse()
            for fileName in picHtmlFileNames :
                logId = re.findall(self.splitLogFIleForNumberRule,fileName);
                if(len(logId)):
                    if(int(logId[0]) > maxLogNumber  or int(logId[0]) < startLogNumber):
                        continue
                else:
                    continue
                if(fileName in picFileNames):
                    print("bak end :",fileName)
                    continue
                else:
                    print("create :",fileName,publishutilitie.getLocationTime())
                    
                asIndexUrlLog = self.log.readLogToList("%s/%s" % tuple([self.picHtmlLogFileDir,fileName]))
                PicUrlLogFile = ("%s/%s" % tuple([self.picLogFileDir,fileName]))
                picUrlList = []
                for urlInfo in asIndexUrlLog :
                    url = re.match('(.*?)\\n',urlInfo)
                    if(len(url.group(0)) != 0) :
                        content = self.getHtmlFromUrl(url.group(0))
                        picUrls = re.findall(self.imgSrcForIdRuel,content,re.VERBOSE)
                        if(len(picUrls) > 0):
                            picUrlList = picUrlList + [picUrls[0]]
                        else:
                            self.log.wirteLog('aaaaaa.log',urlInfo)
                self.log.wirteLog(PicUrlLogFile,picUrlList)

    
    def getHtmlFromUrl(self,url):
        """"""
        downContent = ''
        if(url in self.readUrlList):
            return downContent
        self.readUrlList = self.readUrlList + [url]
        try:
            response = urllib.request.urlopen(url)
            downContent = response.read()
            self.log.wirteLog(self.downLogFile,url)
        except :
            self.log.wirteLog(__name__+'error.log',url)
            return downContent
        if(self.decode):
            downContent = downContent.decode(self.decode)
        return downContent;
    def __del__(self):
        print('down number :' , len(self.readUrlList))
        
if(__name__=='__main__'):
    import configmap
    import sys
    startLogNumber = 0
    if(len(sys.argv)>1):
        try :
            startLogNumber = int(sys.argv[1])
        except :
            print('error startLogNumber ,startLogNumber will ',startLogNumber)
    #logInputDirName = input('Enter log dir name --> ')
    logInputDirName = 'free10'
    treasure(configmap.WebDecode,logInputDirName,False).run(startLogNumber)

